#!/usr/bin/env python
#coding:utf-8
#author:dingyangfan
#url:http://www.meiyear.com

import settings
import MySQLdb,time,socket,urllib2,exceptions
from Queue import Queue,Empty
from threading import Thread
if(settings.insertcharset == 'utf-8'):
    dcode = lambda a:  u'%s'%a
elif (settings.insertcharset == 'gbk'):
    dcode = lambda a: a.encode('gbk','ignore')
socket.setdefaulttimeout(settings.timeout)
q = Queue()
def getttype():
    conn = MySQLdb.Connect(host=settings.mysqlinfo[0],user=settings.mysqlinfo[1],passwd=settings.mysqlinfo[2],db=settings.mysqlinfo[3],charset=settings.mysqlinfo[4],port=settings.mysqlinfo[5])
    c = conn.cursor()
    c.execute("select id,name from ti_type where pid>0")
    tt = c.fetchall()
    c.close()
    conn.close()
    r= {}
    for i in tt:
        r[i[1]] = i[0]
    return r
def db_insert_thread(title,name,typen,ttype,dateline,clicknum,replaynum,content,url,litpic,conn):
    if title is None or name is None or content is None: return 0
    pubdate = int(time.time())
    flag ='p' if litpic !='' else ''
    c = conn.cursor()
    try:
        c.execute("insert into ti_thread (`tid2`,`tid`,`subject`,`dateline`,`name`,`pubdate`,`flag`,`clicknum`,`replaynum`,`url`,`litpic`)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
                (ttype,typen,dcode(title),dateline,dcode(name),pubdate,flag,clicknum,replaynum,url,dcode(litpic))
            )
        lastid = c.lastrowid
        c.execute("insert into ti_threads(`aid`,`tid`,`tid2`,`name`,`dateline`,`body`)values(%s,%s,%s,%s,%s,%s)",(lastid,typen,ttype,dcode(name),dateline,dcode(content)))
    except conn.IntegrityError:
        lastid = 0
    except Exception as e:
        lastid = 0
        print 'thread error type; %s data ; %s'%(typen,e)
    c.close()
    return lastid

def thread_fetch(page,fetchpage,f):
    conn = MySQLdb.Connect(host=settings.mysqlinfo[0],user=settings.mysqlinfo[1],passwd=settings.mysqlinfo[2],db=settings.mysqlinfo[3],charset=settings.mysqlinfo[4],port=settings.mysqlinfo[5])
    while True:
        if q.empty() is True:
            break
        try:
            id = q.get(timeout=1200)
            co = page % id
            logmsg('%s(%s)'%(co,q.qsize()),f)
            fetchpage(co,id,conn)
            q.task_done()
        except exceptions.TypeError:
            logmsg('full Exception',f)
        except Empty:
            logmsg('Empty Exception',f)
    conn.close()
    logmsg('结束队列线程%s'%page,f)
def fetchlist(lists,info):
    sid = 'l_%s'%info['sid']
    for li in lists:
        if info['r'].sismember(sid,li) is False:
            info['r'].sadd(sid,li)
            q.put(li)
def done_start(url,searchindex,fetchlist,info):
    logmsg('采集列表：%s'%url,info['f'])
    li = searchindex(url)
    fetchlist(li,info)
def done_page(page,fetchpage,f,thread_count):
    tt = []
    for i in range(thread_count):
        t = Thread(target=thread_fetch,args=(page,fetchpage,f,))
        tt.append(t)
        #t.setDaemon(True)
        t.start()
    for j in tt:
        j.join()
    logmsg('结束',f)
def vpageurl(p, f,charset = 'utf-8'):
    try:
        content = urllib2.urlopen(p).read().decode(charset,'ignore')
    except urllib2.URLError:
        logmsg('vpage失败2:%s' % p,f)
        return -1
    except Exception:
        logmsg('vpage失败2:%s' % p,f)
        return -1
    logmsg('重试vpage:%s'%p,f)
    return content

def logmsg(msg,filename):
    aa = file('./logs/%s_%s.log'%(filename,time.strftime('%Y-%m-%d',time.gmtime(int(time.time()) + 8*60*60))),'ab')
    print >> aa,msg
    aa.close()